{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyalink.alink import *\n",
    "useLocalEnv(1)\n",
    "\n",
    "from utils import *\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "pd.set_option('display.max_rows', 200)\n",
    "\n",
    "DATA_DIR = ROOT_DIR + \"iris\" + os.sep\n",
    "\n",
    "ORIGIN_FILE = \"iris.data\";\n",
    "VECTOR_FILE = \"iris_vec.ak\";\n",
    "\n",
    "SCHEMA_STRING = \"sepal_length double, sepal_width double, petal_length double, petal_width double, category string\";\n",
    "\n",
    "FEATURE_COL_NAMES = [\"sepal_length\", \"sepal_width\", \"petal_length\", \"petal_width\"]\n",
    "\n",
    "LABEL_COL_NAME = \"category\";\n",
    "VECTOR_COL_NAME = \"vec\";\n",
    "PREDICTION_COL_NAME = \"cluster_id\";\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_1_2\n",
    "if not(os.path.exists(DATA_DIR + VECTOR_FILE)) :\n",
    "    CsvSourceBatchOp()\\\n",
    "        .setFilePath(DATA_DIR + ORIGIN_FILE)\\\n",
    "        .setSchemaStr(SCHEMA_STRING)\\\n",
    "        .link(\n",
    "            VectorAssemblerBatchOp()\\\n",
    "                .setSelectedCols(FEATURE_COL_NAMES)\\\n",
    "                .setOutputCol(VECTOR_COL_NAME)\\\n",
    "                .setReservedCols(LABEL_COL_NAME)\n",
    "        )\\\n",
    "        .link(\n",
    "            AkSinkBatchOp().setFilePath(DATA_DIR + VECTOR_FILE)\n",
    "        );\n",
    "    BatchOperator.execute()\n",
    "\n",
    "source = AkSourceBatchOp().setFilePath(DATA_DIR + VECTOR_FILE);\n",
    "\n",
    "source.lazyPrint(5);\n",
    "\n",
    "kmeans_model = KMeansTrainBatchOp()\\\n",
    "    .setK(2)\\\n",
    "    .setVectorCol(VECTOR_COL_NAME);\n",
    "\n",
    "kmeans_pred = KMeansPredictBatchOp().setPredictionCol(PREDICTION_COL_NAME);\n",
    "\n",
    "source.link(kmeans_model);\n",
    "kmeans_pred.linkFrom(kmeans_model, source);\n",
    "\n",
    "kmeans_model.lazyPrintModelInfo();\n",
    "\n",
    "kmeans_pred.lazyPrint(5);\n",
    "\n",
    "kmeans_pred\\\n",
    "    .link(\n",
    "        EvalClusterBatchOp()\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"KMeans EUCLIDEAN\")\n",
    "    );\n",
    "\n",
    "kmeans_pred\\\n",
    "    .orderBy(PREDICTION_COL_NAME + \", \" + LABEL_COL_NAME, 200)\\\n",
    "    .lazyPrint(-1, \"all data\");\n",
    "\n",
    "BatchOperator.execute()\n",
    "\n",
    "KMeans()\\\n",
    "    .setK(2)\\\n",
    "    .setDistanceType('COSINE')\\\n",
    "    .setVectorCol(VECTOR_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .enableLazyPrintModelInfo()\\\n",
    "    .fit(source)\\\n",
    "    .transform(source)\\\n",
    "    .link(\n",
    "        EvalClusterBatchOp()\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"KMeans COSINE\")\n",
    "    );\n",
    "BatchOperator.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_2_2\n",
    "source = AkSourceBatchOp().setFilePath(DATA_DIR + VECTOR_FILE);\n",
    "\n",
    "GaussianMixture()\\\n",
    "    .setK(2)\\\n",
    "    .setVectorCol(VECTOR_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .enableLazyPrintModelInfo()\\\n",
    "    .fit(source)\\\n",
    "    .transform(source)\\\n",
    "    .link(\n",
    "        EvalClusterBatchOp()\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"GaussianMixture 2\")\n",
    "    );\n",
    "BatchOperator.execute()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_3\n",
    "source = AkSourceBatchOp().setFilePath(DATA_DIR + VECTOR_FILE);\n",
    "\n",
    "BisectingKMeans()\\\n",
    "    .setK(3)\\\n",
    "    .setVectorCol(VECTOR_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .enableLazyPrintModelInfo(\"BiSecting KMeans EUCLIDEAN\")\\\n",
    "    .fit(source)\\\n",
    "    .transform(source)\\\n",
    "    .link(\n",
    "        EvalClusterBatchOp()\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"Bisecting KMeans EUCLIDEAN\")\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "\n",
    "BisectingKMeans()\\\n",
    "    .setDistanceType('COSINE')\\\n",
    "    .setK(3)\\\n",
    "    .setVectorCol(VECTOR_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .enableLazyPrintModelInfo(\"BiSecting KMeans COSINE\")\\\n",
    "    .fit(source)\\\n",
    "    .transform(source)\\\n",
    "    .link(\n",
    "        EvalClusterBatchOp()\\\n",
    "            .setDistanceType(\"COSINE\")\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"Bisecting KMeans COSINE\")\n",
    "    );\n",
    "BatchOperator.execute();\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_4\n",
    "\n",
    "df = pd.DataFrame(\n",
    "    [\n",
    "        [\"Alabama\", \"South\", \"East South Central\", -86.7509, 32.5901],\n",
    "        [\"Alaska\", \"West\", \"Pacific\", -127.25, 49.25],\n",
    "        [\"Arizona\", \"West\", \"Mountain\", -111.625, 34.2192],\n",
    "        [\"Arkansas\", \"South\", \"West South Central\", -92.2992, 34.7336],\n",
    "        [\"California\", \"West\", \"Pacific\", -119.773, 36.5341],\n",
    "        [\"Colorado\", \"West\", \"Mountain\", -105.513, 38.6777],\n",
    "        [\"Connecticut\", \"Northeast\", \"New England\", -72.3573, 41.5928],\n",
    "        [\"Delaware\", \"South\", \"South Atlantic\", -74.9841, 38.6777],\n",
    "        [\"Florida\", \"South\", \"South Atlantic\", -81.685, 27.8744],\n",
    "        [\"Georgia\", \"South\", \"South Atlantic\", -83.3736, 32.3329],\n",
    "        [\"Hawaii\", \"West\", \"Pacific\", -126.25, 31.75],\n",
    "        [\"Idaho\", \"West\", \"Mountain\", -113.93, 43.5648],\n",
    "        [\"Illinois\", \"North Central\", \"East North Central\", -89.3776, 40.0495],\n",
    "        [\"Indiana\", \"North Central\", \"East North Central\", -86.0808, 40.0495],\n",
    "        [\"Iowa\", \"North Central\", \"West North Central\", -93.3714, 41.9358],\n",
    "        [\"Kansas\", \"North Central\", \"West North Central\", -98.1156, 38.4204],\n",
    "        [\"Kentucky\", \"South\", \"East South Central\", -84.7674, 37.3915],\n",
    "        [\"Louisiana\", \"South\", \"West South Central\", -92.2724, 30.6181],\n",
    "        [\"Maine\", \"Northeast\", \"New England\", -68.9801, 45.6226],\n",
    "        [\"Maryland\", \"South\", \"South Atlantic\", -76.6459, 39.2778],\n",
    "        [\"Massachusetts\", \"Northeast\", \"New England\", -71.58, 42.3645],\n",
    "        [\"Michigan\", \"North Central\", \"East North Central\", -84.687, 43.1361],\n",
    "        [\"Minnesota\", \"North Central\", \"West North Central\", -94.6043, 46.3943],\n",
    "        [\"Mississippi\", \"South\", \"East South Central\", -89.8065, 32.6758],\n",
    "        [\"Missouri\", \"North Central\", \"West North Central\", -92.5137, 38.3347],\n",
    "        [\"Montana\", \"West\", \"Mountain\", -109.32, 46.823],\n",
    "        [\"Nebraska\", \"North Central\", \"West North Central\", -99.5898, 41.3356],\n",
    "        [\"Nevada\", \"West\", \"Mountain\", -116.851, 39.1063],\n",
    "        [\"New Hampshire\", \"Northeast\", \"New England\", -71.3924, 43.3934],\n",
    "        [\"New Jersey\", \"Northeast\", \"Middle Atlantic\", -74.2336, 39.9637],\n",
    "        [\"New Mexico\", \"West\", \"Mountain\", -105.942, 34.4764],\n",
    "        [\"New York\", \"Northeast\", \"Middle Atlantic\", -75.1449, 43.1361],\n",
    "        [\"North Carolina\", \"South\", \"South Atlantic\", -78.4686, 35.4195],\n",
    "        [\"North Dakota\", \"North Central\", \"West North Central\", -100.099, 47.2517],\n",
    "        [\"Ohio\", \"North Central\", \"East North Central\", -82.5963, 40.221],\n",
    "        [\"Oklahoma\", \"South\", \"West South Central\", -97.1239, 35.5053],\n",
    "        [\"Oregon\", \"West\", \"Pacific\", -120.068, 43.9078],\n",
    "        [\"Pennsylvania\", \"Northeast\", \"Middle Atlantic\", -77.45, 40.9069],\n",
    "        [\"Rhode Island\", \"Northeast\", \"New England\", -71.1244, 41.5928],\n",
    "        [\"South Carolina\", \"South\", \"South Atlantic\", -80.5056, 33.619],\n",
    "        [\"South Dakota\", \"North Central\", \"West North Central\", -99.7238, 44.3365],\n",
    "        [\"Tennessee\", \"South\", \"East South Central\", -86.456, 35.6767],\n",
    "        [\"Texas\", \"South\", \"West South Central\", -98.7857, 31.3897],\n",
    "        [\"Utah\", \"West\", \"Mountain\", -111.33, 39.1063],\n",
    "        [\"Vermont\", \"Northeast\", \"New England\", -72.545, 44.2508],\n",
    "        [\"Virginia\", \"South\", \"South Atlantic\", -78.2005, 37.563],\n",
    "        [\"Washington\", \"West\", \"Pacific\", -119.746, 47.4231],\n",
    "        [\"West Virginia\", \"South\", \"South Atlantic\", -80.6665, 38.4204],\n",
    "        [\"Wisconsin\", \"North Central\", \"East North Central\", -89.9941, 44.5937],\n",
    "        [\"Wyoming\", \"West\", \"Mountain\", -107.256, 43.0504]\n",
    "    ]\n",
    ")\n",
    "schema_str = \"State string, Region string, Division string, longitude double, latitude double\"\n",
    "source = BatchOperator.fromDataframe(df, schema_str)\n",
    "\n",
    "source.lazyPrint(5);\n",
    "\n",
    "source.select(\"Region\").distinct().lazyPrint(-1);\n",
    "\n",
    "source.select(\"Division\").distinct().lazyPrint(-1);\n",
    "\n",
    "source\\\n",
    "    .groupBy(\"Region, Division\", \"Region, Division, COUNT(*) AS numStates\")\\\n",
    "    .orderBy(\"Region, Division\", 100)\\\n",
    "    .lazyPrint(-1);\n",
    "\n",
    "for nClusters in [2, 4] :\n",
    "    pred = GeoKMeans()\\\n",
    "        .setLongitudeCol(\"longitude\")\\\n",
    "        .setLatitudeCol(\"latitude\")\\\n",
    "        .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "        .setK(nClusters)\\\n",
    "        .fit(source)\\\n",
    "        .transform(source);\n",
    "\n",
    "    pred.link(\n",
    "        EvalClusterBatchOp()\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .setLabelCol(\"Region\")\\\n",
    "            .lazyPrintMetrics(str(nClusters) + \" with Region\")\n",
    "    );\n",
    "    pred.link(\n",
    "        EvalClusterBatchOp()\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .setLabelCol(\"Division\")\\\n",
    "            .lazyPrintMetrics(str(nClusters) + \" with Division\")\n",
    "    );\n",
    "    BatchOperator.execute()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
